{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# example of a super learner model for regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train (500, 100) (500,) Test (500, 100) (500,)\n",
      "Meta  (500, 9) (500,)\n",
      "LinearRegression: RMSE 0.570\n",
      "ElasticNet: RMSE 59.888\n",
      "SVR: RMSE 154.986\n",
      "DecisionTreeRegressor: RMSE 130.260\n",
      "KNeighborsRegressor: RMSE 136.732\n",
      "AdaBoostRegressor: RMSE 80.056\n",
      "BaggingRegressor: RMSE 85.996\n",
      "RandomForestRegressor: RMSE 87.410\n",
      "ExtraTreesRegressor: RMSE 86.981\n",
      "Super Learner: RMSE 0.572\n"
     ]
    }
   ],
   "source": [
    "from math import sqrt\n",
    "from numpy import hstack\n",
    "from numpy import vstack\n",
    "from numpy import asarray\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import ElasticNet\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "from sklearn.ensemble import BaggingRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import ExtraTreesRegressor\n",
    "\n",
    "# create a list of base-models\n",
    "def get_models():\n",
    "    models = list()\n",
    "    models.append(LinearRegression())\n",
    "    models.append(ElasticNet())\n",
    "    models.append(SVR(gamma='scale'))\n",
    "    models.append(DecisionTreeRegressor())\n",
    "    models.append(KNeighborsRegressor())\n",
    "    models.append(AdaBoostRegressor())\n",
    "    models.append(BaggingRegressor(n_estimators=10))\n",
    "    models.append(RandomForestRegressor(n_estimators=10))\n",
    "    models.append(ExtraTreesRegressor(n_estimators=10))\n",
    "    return models\n",
    "\n",
    "# collect out of fold predictions form k-fold cross validation\n",
    "def get_out_of_fold_predictions(X, y, models):\n",
    "\tmeta_X, meta_y = list(), list()\n",
    "\t# define split of data\n",
    "\tkfold = KFold(n_splits=10, shuffle=True)\n",
    "\t# enumerate splits\n",
    "\tfor train_ix, test_ix in kfold.split(X):\n",
    "\t\tfold_yhats = list()\n",
    "\t\t# get data\n",
    "\t\ttrain_X, test_X = X[train_ix], X[test_ix]\n",
    "\t\ttrain_y, test_y = y[train_ix], y[test_ix]\n",
    "\t\tmeta_y.extend(test_y)\n",
    "\t\t# fit and make predictions with each sub-model\n",
    "\t\tfor model in models:\n",
    "\t\t\tmodel.fit(train_X, train_y)\n",
    "\t\t\tyhat = model.predict(test_X)\n",
    "\t\t\t# store columns\n",
    "\t\t\tfold_yhats.append(yhat.reshape(len(yhat),1))\n",
    "\t\t# store fold yhats as columns\n",
    "\t\tmeta_X.append(hstack(fold_yhats))\n",
    "\treturn vstack(meta_X), asarray(meta_y)\n",
    "\n",
    "# fit all base models on the training dataset\n",
    "def fit_base_models(X, y, models):\n",
    "\tfor model in models:\n",
    "\t\tmodel.fit(X, y)\n",
    "\n",
    "# fit a meta model\n",
    "def fit_meta_model(X, y):\n",
    "\tmodel = LinearRegression()\n",
    "\tmodel.fit(X, y)\n",
    "\treturn model\n",
    "\n",
    "# evaluate a list of models on a dataset\n",
    "def evaluate_models(X, y, models):\n",
    "\tfor model in models:\n",
    "\t\tyhat = model.predict(X)\n",
    "\t\tmse = mean_squared_error(y, yhat)\n",
    "\t\tprint('%s: RMSE %.3f' % (model.__class__.__name__, sqrt(mse)))\n",
    "\n",
    "# make predictions with stacked model\n",
    "def super_learner_predictions(X, models, meta_model):\n",
    "\tmeta_X = list()\n",
    "\tfor model in models:\n",
    "\t\tyhat = model.predict(X)\n",
    "\t\tmeta_X.append(yhat.reshape(len(yhat),1))\n",
    "\tmeta_X = hstack(meta_X)\n",
    "\t# predict\n",
    "\treturn meta_model.predict(meta_X)\n",
    "\n",
    "# create the inputs and outputs\n",
    "X, y = make_regression(n_samples=1000, n_features=100, noise=0.5)\n",
    "# split\n",
    "X, X_val, y, y_val = train_test_split(X, y, test_size=0.50)\n",
    "print('Train', X.shape, y.shape, 'Test', X_val.shape, y_val.shape)\n",
    "# get models\n",
    "models = get_models()\n",
    "# get out of fold predictions\n",
    "meta_X, meta_y = get_out_of_fold_predictions(X, y, models)\n",
    "print('Meta ', meta_X.shape, meta_y.shape)\n",
    "# fit base models\n",
    "fit_base_models(X, y, models)\n",
    "# fit the meta model\n",
    "meta_model = fit_meta_model(meta_X, meta_y)\n",
    "# evaluate base models\n",
    "evaluate_models(X_val, y_val, models)\n",
    "# evaluate meta model\n",
    "yhat = super_learner_predictions(X_val, models, meta_model)\n",
    "print('Super Learner: RMSE %.3f' % (sqrt(mean_squared_error(y_val, yhat))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# example of a super learner for regression using the mlens library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[MLENS] backend: threading\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train (500, 100) (500,) Test (500, 100) (500,)\n",
      "                                  score-m  score-s  ft-m  ft-s  pt-m  pt-s\n",
      "layer-1  adaboostregressor          84.11     7.52  2.15  0.02  0.08  0.02\n",
      "layer-1  baggingregressor           95.26     6.67  0.52  0.10  0.03  0.00\n",
      "layer-1  decisiontreeregressor     132.30    11.77  0.09  0.04  0.00  0.01\n",
      "layer-1  elasticnet                 62.36     7.45  0.06  0.03  0.01  0.01\n",
      "layer-1  extratreesregressor        86.42     8.45  0.56  0.10  0.02  0.01\n",
      "layer-1  kneighborsregressor       136.36    12.09  0.01  0.01  0.02  0.01\n",
      "layer-1  linearregression            0.57     0.05  0.07  0.03  0.01  0.01\n",
      "layer-1  randomforestregressor      92.35     5.57  0.46  0.17  0.00  0.00\n",
      "layer-1  svr                       154.08    11.80  0.04  0.01  0.00  0.00\n",
      "\n",
      "Super Learner: RMSE 0.534\n"
     ]
    }
   ],
   "source": [
    "from math import sqrt\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import ElasticNet\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "from sklearn.ensemble import BaggingRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import ExtraTreesRegressor\n",
    "from mlens.ensemble import SuperLearner\n",
    "\n",
    "# create a list of base-models\n",
    "def get_models():\n",
    "\tmodels = list()\n",
    "\tmodels.append(LinearRegression())\n",
    "\tmodels.append(ElasticNet())\n",
    "\tmodels.append(SVR(gamma='scale'))\n",
    "\tmodels.append(DecisionTreeRegressor())\n",
    "\tmodels.append(KNeighborsRegressor())\n",
    "\tmodels.append(AdaBoostRegressor())\n",
    "\tmodels.append(BaggingRegressor(n_estimators=10))\n",
    "\tmodels.append(RandomForestRegressor(n_estimators=10))\n",
    "\tmodels.append(ExtraTreesRegressor(n_estimators=10))\n",
    "\treturn models\n",
    "\n",
    "# cost function for base models\n",
    "def rmse(yreal, yhat):\n",
    "\treturn sqrt(mean_squared_error(yreal, yhat))\n",
    "\n",
    "# create the super learner\n",
    "def get_super_learner(X):\n",
    "\tensemble = SuperLearner(scorer=rmse, folds=10, shuffle=True, sample_size=len(X))\n",
    "\t# add base models\n",
    "\tmodels = get_models()\n",
    "\tensemble.add(models)\n",
    "\t# add the meta model\n",
    "\tensemble.add_meta(LinearRegression())\n",
    "\treturn ensemble\n",
    "\n",
    "# create the inputs and outputs\n",
    "X, y = make_regression(n_samples=1000, n_features=100, noise=0.5)\n",
    "# split\n",
    "X, X_val, y, y_val = train_test_split(X, y, test_size=0.50)\n",
    "print('Train', X.shape, y.shape, 'Test', X_val.shape, y_val.shape)\n",
    "# create the super learner\n",
    "ensemble = get_super_learner(X)\n",
    "# fit the super learner\n",
    "ensemble.fit(X, y)\n",
    "# summarize base learners\n",
    "print(ensemble.data)\n",
    "# evaluate meta model\n",
    "yhat = ensemble.predict(X_val)\n",
    "print('Super Learner: RMSE %.3f' % (rmse(y_val, yhat)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# example of a super learner using the mlens library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train (500, 100) (500,) Test (500, 100) (500,)\n",
      "                                   score-m  score-s  ft-m  ft-s  pt-m  pt-s\n",
      "layer-1  adaboostclassifier           0.91     0.04  2.97  0.13  0.19  0.06\n",
      "layer-1  baggingclassifier            0.81     0.05  0.74  0.13  0.07  0.02\n",
      "layer-1  decisiontreeclassifier       0.74     0.04  0.07  0.01  0.00  0.00\n",
      "layer-1  extratreesclassifier         0.83     0.02  0.72  0.13  0.03  0.01\n",
      "layer-1  gaussiannb                   0.98     0.01  0.05  0.01  0.02  0.01\n",
      "layer-1  kneighborsclassifier         0.93     0.03  0.01  0.01  0.13  0.02\n",
      "layer-1  logisticregression           0.93     0.04  0.03  0.01  0.01  0.00\n",
      "layer-1  randomforestclassifier       0.83     0.06  0.75  0.20  0.01  0.01\n",
      "layer-1  svc                          0.97     0.01  0.13  0.01  0.00  0.00\n",
      "\n",
      "Super Learner: 98.000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import make_blobs\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from mlens.ensemble import SuperLearner\n",
    "\n",
    "# create a list of base-models\n",
    "def get_models():\n",
    "\tmodels = list()\n",
    "\tmodels.append(LogisticRegression(solver='liblinear'))\n",
    "\tmodels.append(DecisionTreeClassifier())\n",
    "\tmodels.append(SVC(gamma='scale', probability=True))\n",
    "\tmodels.append(GaussianNB())\n",
    "\tmodels.append(KNeighborsClassifier())\n",
    "\tmodels.append(AdaBoostClassifier())\n",
    "\tmodels.append(BaggingClassifier(n_estimators=10))\n",
    "\tmodels.append(RandomForestClassifier(n_estimators=10))\n",
    "\tmodels.append(ExtraTreesClassifier(n_estimators=10))\n",
    "\treturn models\n",
    "\n",
    "# create the super learner\n",
    "def get_super_learner(X):\n",
    "\tensemble = SuperLearner(scorer=accuracy_score, folds=10, shuffle=True, sample_size=len(X))\n",
    "\t# add base models\n",
    "\tmodels = get_models()\n",
    "\tensemble.add(models)\n",
    "\t# add the meta model\n",
    "\tensemble.add_meta(LogisticRegression(solver='lbfgs'))\n",
    "\treturn ensemble\n",
    "\n",
    "# create the inputs and outputs\n",
    "X, y = make_blobs(n_samples=1000, centers=2, n_features=100, cluster_std=20)\n",
    "# split\n",
    "X, X_val, y, y_val = train_test_split(X, y, test_size=0.50)\n",
    "print('Train', X.shape, y.shape, 'Test', X_val.shape, y_val.shape)\n",
    "# create the super learner\n",
    "ensemble = get_super_learner(X)\n",
    "# fit the super learner\n",
    "ensemble.fit(X, y)\n",
    "# summarize base learners\n",
    "print(ensemble.data)\n",
    "# make predictions on hold out set\n",
    "yhat = ensemble.predict(X_val)\n",
    "print('Super Learner: %.3f' % (accuracy_score(y_val, yhat) * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pt",
   "language": "python",
   "name": "pt"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
